accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject [38/39] accumulo git commit: ACCUMULO-4173: Updated with changes based on testing
Date Tue, 12 Apr 2016 13:58:55 GMT
ACCUMULO-4173: Updated with changes based on testing


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d745e7c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d745e7c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d745e7c1

Branch: refs/heads/ACCUMULO-4173
Commit: d745e7c1e9baf333f4a54f5ceeac22a588a927d5
Parents: c77a577
Author: Dave Marion <dlmarion@apache.org>
Authored: Tue Apr 12 09:47:44 2016 -0400
Committer: Dave Marion <dlmarion@apache.org>
Committed: Tue Apr 12 09:47:44 2016 -0400

----------------------------------------------------------------------
 .../balancer/HostRegexTableLoadBalancer.java    | 129 +++++++++++--------
 .../HostRegexTableLoadBalancerTest.java         |  48 ++++---
 2 files changed, 101 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/d745e7c1/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 677e4ba..9d92361 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  * This balancer will create pools of tablet servers by grouping tablet servers that match
a regex into the same pool and calling the balancer set on the table
- * to balance within the set of matching tablet servers. <br>
+ * to balance within the set of matching tablet servers. All tablet servers that do not match
a regex are grouped into a default pool.<br>
  * Regex properties for this balancer are specified as:<br>
  * <b>table.custom.balancer.host.regex.&lt;tablename&gt;=&lt;regex&gt;</b><br>
  * Periodically (default 5m) this balancer will check to see if a tablet server is hosting
tablets that it should not be according to the regex configuration.
@@ -75,8 +75,8 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
 
   private Map<String,String> tableIdToTableName = null;
   private Map<String,Pattern> poolNameToRegexPattern = null;
-  private long lastOOBCheck = System.currentTimeMillis();
-  private long lastPoolRecheck = 0;
+  private volatile long lastOOBCheck = System.currentTimeMillis();
+  private volatile long lastPoolRecheck = 0;
   private boolean isIpBasedRegex = false;
   private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new
HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
 
@@ -89,6 +89,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
    */
   protected synchronized Map<String,SortedMap<TServerInstance,TabletServerStatus>>
splitCurrentByRegex(SortedMap<TServerInstance,TabletServerStatus> current) {
     if ((System.currentTimeMillis() - lastPoolRecheck) > poolRecheckMillis) {
+      LOG.debug("Performing pool recheck - regrouping tablet servers into pools based on
regex");
       Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools = new
HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
       for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
         List<String> poolNames = getPoolNamesForHost(e.getKey().host());
@@ -168,9 +169,11 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
       Map<String,String> customProps = conf.getTableConfiguration(table.getValue()).getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
       if (null != customProps && customProps.size() > 0) {
         for (Entry<String,String> customProp : customProps.entrySet()) {
-          String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
-          String regex = customProp.getValue();
-          poolNameToRegexPattern.put(tableName, Pattern.compile(regex));
+          if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
+            String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
+            String regex = customProp.getValue();
+            poolNameToRegexPattern.put(tableName, Pattern.compile(regex));
+          }
         }
       }
     }
@@ -218,40 +221,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
   public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current,
Map<KeyExtent,TServerInstance> unassigned,
       Map<KeyExtent,TServerInstance> assignments) {
 
-    Map<KeyExtent,TServerInstance> unassignedClone = new TreeMap<>(unassigned);
-    if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
-      // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate
it.
-      for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
-        for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
-          for (String table : poolNameToRegexPattern.keySet()) {
-            // pool names are the same as table names, except in the DEFAULT case.
-            if (assignedPool.equals(table) || table.equals(DEFAULT_POOL)) {
-              continue;
-            }
-            String tid = getTableOperations().tableIdMap().get(table);
-            if (null == tid) {
-              LOG.warn("Unable to check for out of bounds tablets for table {}, it may have
been deleted or renamed.", table);
-              continue;
-            }
-            try {
-              List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(),
tid);
-              for (TabletStats ts : outOfBoundsTablets) {
-                LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning",
ts.toString());
-                unassignedClone.put(new KeyExtent(ts.getExtent()), e.getKey());
-              }
-            } catch (TException e1) {
-              LOG.error("Error in OOB check getting tablets for table {} from server {}",
tid, e.getKey().host(), e);
-            }
-          }
-        }
-      }
-      this.oobCheckMillis = System.currentTimeMillis();
-    }
-
     Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = splitCurrentByRegex(current);
-    // separate the unassigned into tables
+    // group the unassigned into tables
     Map<String,Map<KeyExtent,TServerInstance>> groupedUnassigned = new HashMap<String,Map<KeyExtent,TServerInstance>>();
-    for (Entry<KeyExtent,TServerInstance> e : unassignedClone.entrySet()) {
+    for (Entry<KeyExtent,TServerInstance> e : unassigned.entrySet()) {
       Map<KeyExtent,TServerInstance> tableUnassigned = groupedUnassigned.get(e.getKey().getTableId());
       if (tableUnassigned == null) {
         tableUnassigned = new HashMap<KeyExtent,TServerInstance>();
@@ -259,29 +232,20 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
       }
       tableUnassigned.put(e.getKey(), e.getValue());
     }
-    // Validate that the last server in the unassigned map is actually in the pool, if not
set to null.
-    Map<String,KeyExtent> overwrites = new TreeMap<>();
-    for (Entry<String,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet())
{
-      String poolName = tableIdToTableName.get(e.getKey());
-      for (Entry<KeyExtent,TServerInstance> e2 : e.getValue().entrySet()) {
-        if (null != e2.getValue()) {
-          List<String> hostPools = getPoolNamesForHost(e2.getValue().host());
-          if (!hostPools.contains(poolName)) {
-            overwrites.put(e.getKey(), e2.getKey());
-          }
-        }
-      }
-    }
-    for (Entry<String,KeyExtent> e : overwrites.entrySet()) {
-      groupedUnassigned.get(e.getKey()).remove(e.getValue());
-      groupedUnassigned.get(e.getKey()).put(e.getValue(), null);
-    }
     // Send a view of the current servers to the tables tablet balancer
     for (Entry<String,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet())
{
       Map<KeyExtent,TServerInstance> newAssignments = new HashMap<KeyExtent,TServerInstance>();
       String tableName = tableIdToTableName.get(e.getKey());
       String poolName = getPoolNameForTable(tableName);
       SortedMap<TServerInstance,TabletServerStatus> currentView = pools.get(poolName);
+      if (null == currentView || currentView.size() == 0) {
+        LOG.warn("No tablet servers online for table {}, assigning within default pool",
tableName);
+        currentView = pools.get(DEFAULT_POOL);
+        if (null == currentView) {
+          LOG.error("No tablet servers exist in the default pool, unable to assign tablets
for table {}", tableName);
+        }
+      }
+      LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers
{}", e.getValue().size(), tableName, currentView.keySet());
       getBalancerForTable(e.getKey()).getAssignments(currentView, e.getValue(), newAssignments);
       assignments.putAll(newAssignments);
     }
@@ -295,16 +259,67 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
     if (t == null)
       return minBalanceTime;
 
+    Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped
= splitCurrentByRegex(current);
+    if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
+      try {
+        // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate
it.
+        for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+          for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
+            for (String table : poolNameToRegexPattern.keySet()) {
+              // pool names are the same as table names, except in the DEFAULT case.
+              if (assignedPool.equals(table)) {
+                // If this tserver is assigned to a regex pool, then we can skip checking
tablets for this table on this host.
+                continue;
+              }
+              String tid = getTableOperations().tableIdMap().get(table);
+              if (null == tid) {
+                LOG.warn("Unable to check for out of bounds tablets for table {}, it may
have been deleted or renamed.", table);
+                continue;
+              }
+              try {
+                List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(),
tid);
+                for (TabletStats ts : outOfBoundsTablets) {
+                  KeyExtent ke = new KeyExtent(ts.getExtent());
+                  if (migrations.contains(ke)) {
+                    LOG.debug("Migration for  out of bounds tablet {} has already been requested",
ke);
+                    ;
+                    continue;
+                  }
+                  String poolName = getPoolNameForTable(table);
+                  SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(poolName);
+                  if (null != currentView) {
+                    TServerInstance nextTS = currentView.firstKey();
+                    LOG.info("Tablet {} is currently outside the bounds of the regex, migrating
from {} to {}", ke, e.getKey(), nextTS);
+                    migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
+                  } else {
+                    LOG.warn("No tablet servers online for pool {}, unable to migrate out
of bounds tablets", poolName);
+                  }
+                }
+              } catch (TException e1) {
+                LOG.error("Error in OOB check getting tablets for table {} from server {}",
tid, e.getKey().host(), e);
+              }
+            }
+          }
+        }
+      } finally {
+        this.lastOOBCheck = System.currentTimeMillis();
+      }
+    }
+
     if (migrations != null && migrations.size() > 0) {
-      LOG.warn("Not balancing tables due to outstanding migrations");
+      LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
       return minBalanceTime;
     }
 
-    Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped
= splitCurrentByRegex(current);
     for (String s : t.tableIdMap().values()) {
       String tableName = tableIdToTableName.get(s);
       String regexTableName = getPoolNameForTable(tableName);
       SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(regexTableName);
+      if (null == currentView) {
+        LOG.warn("Skipping balance for table {} as no tablet servers are online, will recheck
for online tservers at {} ms intervals", tableName,
+            this.poolRecheckMillis);
+        continue;
+      }
       ArrayList<TabletMigration> newMigrations = new ArrayList<TabletMigration>();
       long tableBalanceTime = getBalancerForTable(s).balance(currentView, migrations, newMigrations);
       if (tableBalanceTime < minBalanceTime) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d745e7c1/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
index 22bda0b..b273c5b 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
@@ -21,9 +21,11 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
@@ -548,30 +550,27 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer
{
   }
 
   @Test
-  public void testOutOfBoundsTablets() {
+  public void testUnassignedWithNoTServers() {
     init((ServerConfiguration) factory);
-    // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
-    UtilWaitThread.sleep(11000);
     Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
     Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
-    for (List<KeyExtent> extents : tableExtents.values()) {
-      for (KeyExtent ke : extents) {
-        unassigned.put(ke, null);
-      }
+    for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
+      unassigned.put(ke, null);
     }
-    this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned),
assignments);
-    Assert.assertEquals(15, assignments.size());
-    // Ensure unique tservers
-    for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
-      for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
-        if (e.getKey().equals(e2.getKey())) {
-          continue;
-        }
-        if (e.getValue().equals(e2.getValue())) {
-          Assert.fail("Assignment failure");
-        }
+    SortedMap<TServerInstance,TabletServerStatus> current = createCurrent(15);
+    // Remove the BAR tablet servers from current
+    List<TServerInstance> removals = new ArrayList<TServerInstance>();
+    for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+      if (e.getKey().host().equals("192.168.0.6") || e.getKey().host().equals("192.168.0.7")
|| e.getKey().host().equals("192.168.0.8")
+          || e.getKey().host().equals("192.168.0.9") || e.getKey().host().equals("192.168.0.10"))
{
+        removals.add(e.getKey());
       }
     }
+    for (TServerInstance r : removals) {
+      current.remove(r);
+    }
+    this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned),
assignments);
+    Assert.assertEquals(unassigned.size(), assignments.size());
     // Ensure assignments are correct
     for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
       if (!tabletInBounds(e.getKey(), e.getValue())) {
@@ -580,9 +579,20 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer
{
     }
   }
 
+  @Test
+  public void testOutOfBoundsTablets() {
+    init((ServerConfiguration) factory);
+    // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
+    UtilWaitThread.sleep(11000);
+    Set<KeyExtent> migrations = new HashSet<KeyExtent>();
+    List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+    this.balance(createCurrent(15), migrations, migrationsOut);
+    Assert.assertEquals(2, migrationsOut.size());
+  }
+
   @Override
   public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String
tableId) throws ThriftSecurityException, TException {
-    // Report incorrent information so that getAssignments will create n assignment
+    // Report incorrect information so that balance will create an assignment
     List<TabletStats> tablets = new ArrayList<>();
     if (tableId.equals(BAR.getId()) && tserver.host().equals("192.168.0.1")) {
       // Report that we have a bar tablet on this server


Mime
View raw message