accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/2] git commit: ACCUMULO-2952 move tablets to multiple destinations in one pass
Date Mon, 30 Jun 2014 15:23:41 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master dbb9cc4a2 -> 1b89892da


ACCUMULO-2952 move tablets to multiple destinations in one pass


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1425450e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1425450e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1425450e

Branch: refs/heads/master
Commit: 1425450e91b90895537e98a226628d55ca9b64f9
Parents: dbb9cc4
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon Jun 30 11:17:08 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon Jun 30 11:20:00 2014 -0400

----------------------------------------------------------------------
 .../master/balancer/DefaultLoadBalancer.java    | 21 +++--
 .../balancer/DefaultLoadBalancerTest.java       |  2 +-
 .../org/apache/accumulo/test/BalanceFaster.java | 80 ++++++++++++++++++++
 3 files changed, 94 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
index 1fcab46..46b9b5f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
@@ -110,6 +110,7 @@ public class DefaultLoadBalancer extends TabletBalancer {
       if (current.size() < 2) {
         return false;
       }
+      final Map<String,Map<KeyExtent,TabletStats>> donerTabletStats = new HashMap<String,Map<KeyExtent,TabletStats>>();
       
       // Sort by total number of online tablets, per server
       int total = 0;
@@ -143,7 +144,8 @@ public class DefaultLoadBalancer extends TabletBalancer {
       // under-loaded servers.
       int end = totals.size() - 1;
       int movedAlready = 0;
-      for (int tooManyIndex = 0; tooManyIndex < totals.size(); tooManyIndex++) {
+      int tooManyIndex = 0;
+      while (tooManyIndex < totals.size() && end > tooManyIndex) {
         ServerCounts tooMany = totals.get(tooManyIndex);
         int goal = even;
         if (tooManyIndex < numServersOverEven) {
@@ -156,15 +158,19 @@ public class DefaultLoadBalancer extends TabletBalancer {
           break;
         }
         if (needToUnload >= needToLoad) {
-          result.addAll(move(tooMany, tooLittle, needToLoad));
+          result.addAll(move(tooMany, tooLittle, needToLoad, donerTabletStats));
           end--;
           movedAlready = 0;
         } else {
-          result.addAll(move(tooMany, tooLittle, needToUnload));
+          result.addAll(move(tooMany, tooLittle, needToUnload, donerTabletStats));
           movedAlready += needToUnload;
         }
-        if (needToUnload > needToLoad)
+        if (needToUnload > needToLoad) {
           moreBalancingNeeded = true;
+        } else {
+          tooManyIndex++;
+          donerTabletStats.clear();
+        }
       }
       
     } finally {
@@ -186,13 +192,12 @@ public class DefaultLoadBalancer extends TabletBalancer {
   /**
    * Select a tablet based on differences between table loads; if the loads are even, use
the busiest table
    */
-  List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count)
{
+  List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count,
Map<String,Map<KeyExtent,TabletStats>> donerTabletStats) {
     
     List<TabletMigration> result = new ArrayList<TabletMigration>();
     if (count == 0)
       return result;
     
-    Map<String,Map<KeyExtent,TabletStats>> onlineTablets = new HashMap<String,Map<KeyExtent,TabletStats>>();
     // Copy counts so we can update them as we propose migrations
     Map<String,Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
     Map<String,Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
@@ -224,13 +229,13 @@ public class DefaultLoadBalancer extends TabletBalancer {
         // just balance the given table
         table = tableToBalance;
       }
-      Map<KeyExtent,TabletStats> onlineTabletsForTable = onlineTablets.get(table);
+      Map<KeyExtent,TabletStats> onlineTabletsForTable = donerTabletStats.get(table);
       try {
         if (onlineTabletsForTable == null) {
           onlineTabletsForTable = new HashMap<KeyExtent,TabletStats>();
           for (TabletStats stat : getOnlineTabletsForTable(tooMuch.server, table))
             onlineTabletsForTable.put(new KeyExtent(stat.extent), stat);
-          onlineTablets.put(table, onlineTabletsForTable);
+          donerTabletStats.put(table, onlineTabletsForTable);
         }
       } catch (Exception ex) {
         log.error("Unable to select a tablet to move", ex);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
index 9f99b1c..0439429 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
@@ -203,7 +203,7 @@ public class DefaultLoadBalancerTest {
         servers.get(migration.newServer).extents.add(migration.tablet);
       }
     }
-    assertEquals(8, moved);
+    assertEquals(9, moved);
   }
   
   @Test

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1425450e/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java b/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
new file mode 100644
index 0000000..a6fe5d3
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/BalanceFaster.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-2952
+public class BalanceFaster extends ConfigurableMacIT {
+  
+  @Override
+  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+    cfg.setNumTservers(3);
+  }
+
+  @Test(timeout=30*1000)
+  public void test() throws Exception {
+    String tableName = getUniqueNames(1)[0];
+    Connector conn = getConnector();
+    conn.tableOperations().create(tableName);
+    SortedSet<Text> splits = new TreeSet<Text>();
+    for (int i = 0; i < 1000; i++) {
+      splits.add(new Text("" + i));
+    }
+    conn.tableOperations().addSplits(tableName, splits);
+    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    UtilWaitThread.sleep(5000);
+    s.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+    s.setRange(MetadataSchema.TabletsSection.getRange());
+    Map<String, Integer> counts = new HashMap<String, Integer>();
+    for (Entry<Key,Value> kv : s) {
+      String host = kv.getValue().toString();
+      if (!counts.containsKey(host))
+        counts.put(host, 0);
+      counts.put(host, counts.get(host) + 1);
+    }
+    assertTrue(counts.size() == 3);
+    Iterator<Integer> i = counts.values().iterator();
+    int a = i.next();
+    int b = i.next();
+    int c = i.next();
+    assertTrue(Math.abs(a - b) < 3);
+    assertTrue(Math.abs(a - c) < 3);
+  }
+  
+}


Mime
View raw message