accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject [2/2] accumulo git commit: Merge branch '1.7'
Date Thu, 14 Apr 2016 20:13:51 GMT
Merge branch '1.7'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ef39c1aa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ef39c1aa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ef39c1aa

Branch: refs/heads/master
Commit: ef39c1aaeadb3b12cd90262f5c6f85e2d02a719d
Parents: 584b812 c37ba87
Author: Dave Marion <dlmarion@apache.org>
Authored: Thu Apr 14 16:12:42 2016 -0400
Committer: Dave Marion <dlmarion@apache.org>
Committed: Thu Apr 14 16:12:42 2016 -0400

----------------------------------------------------------------------
 .../balancer/HostRegexTableLoadBalancer.java    | 136 ++++++---
 .../BaseHostRegexTableLoadBalancerTest.java     | 289 +++++++++++++++++++
 ...gexTableLoadBalancerReconfigurationTest.java | 106 +++++++
 .../HostRegexTableLoadBalancerTest.java         | 259 +----------------
 4 files changed, 498 insertions(+), 292 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/ef39c1aa/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ef39c1aa/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
index 0000000,aa1480f..4ac08d2
mode 000000,100644..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
@@@ -1,0 -1,289 +1,289 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.accumulo.server.master.balancer;
+ 
+ import java.net.UnknownHostException;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.SortedMap;
+ import java.util.TreeMap;
+ import java.util.Map.Entry;
+ 
+ import org.apache.accumulo.core.client.AccumuloException;
+ import org.apache.accumulo.core.client.AccumuloSecurityException;
+ import org.apache.accumulo.core.client.Connector;
+ import org.apache.accumulo.core.client.Instance;
+ import org.apache.accumulo.core.client.admin.TableOperations;
+ import org.apache.accumulo.core.client.impl.ClientContext;
+ import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+ import org.apache.accumulo.core.conf.AccumuloConfiguration;
+ import org.apache.accumulo.core.conf.ConfigurationCopy;
+ import org.apache.accumulo.core.conf.Property;
+ import org.apache.accumulo.core.data.impl.KeyExtent;
+ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+ import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+ import org.apache.accumulo.server.conf.TableConfiguration;
+ import org.apache.accumulo.server.master.state.TServerInstance;
+ import org.apache.hadoop.io.Text;
+ import org.easymock.EasyMock;
+ 
+ import com.google.common.base.Predicate;
+ 
+ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer
{
+ 
+   protected static class TestInstance implements Instance {
+ 
+     @Override
+     public String getRootTabletLocation() {
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public List<String> getMasterLocations() {
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public String getInstanceID() {
+       return "1111";
+     }
+ 
+     @Override
+     public String getInstanceName() {
+       return "test";
+     }
+ 
+     @Override
+     public String getZooKeepers() {
+       return "";
+     }
+ 
+     @Override
+     public int getZooKeepersSessionTimeOut() {
+       return 30;
+     }
+ 
+     @Override
+     public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException
{
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException,
AccumuloSecurityException {
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public Connector getConnector(String user, CharSequence pass) throws AccumuloException,
AccumuloSecurityException {
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public AccumuloConfiguration getConfiguration() {
+       throw new UnsupportedOperationException();
+     }
+ 
+     @Override
+     public void setConfiguration(AccumuloConfiguration conf) {}
+ 
+     @Override
+     public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException,
AccumuloSecurityException {
+       throw new UnsupportedOperationException();
+     }
+ 
+   }
+ 
+   protected static class Table {
+     private String tableName;
+     private String id;
+ 
+     Table(String tableName, String id) {
+       this.tableName = tableName;
+       this.id = id;
+     }
+ 
+     public String getTableName() {
+       return tableName;
+     }
+ 
+     public String getId() {
+       return id;
+     }
+   }
+ 
+   protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>();
+   {
+     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY,
"10s");
+     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY,
"30s");
+     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(),
"r01.*");
+     DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(),
"r02.*");
+   }
+ 
+   protected static class TestServerConfigurationFactory extends ServerConfigurationFactory
{
+ 
+     public TestServerConfigurationFactory(Instance instance) {
+       super(instance);
+     }
+ 
+     @Override
+     public synchronized AccumuloConfiguration getConfiguration() {
+       return new ConfigurationCopy(DEFAULT_TABLE_PROPERTIES);
+     }
+ 
+     @Override
+     public TableConfiguration getTableConfiguration(String tableId) {
+       return new TableConfiguration(getInstance(), tableId, null) {
+         @Override
+         public String get(Property property) {
+           return DEFAULT_TABLE_PROPERTIES.get(property.name());
+         }
+ 
+         @Override
+         public void getProperties(Map<String,String> props, Predicate<String>
filter) {
+           for (Entry<String,String> e : DEFAULT_TABLE_PROPERTIES.entrySet()) {
+             if (filter.apply(e.getKey())) {
+               props.put(e.getKey(), e.getValue());
+             }
+           }
+         }
+       };
+     }
+   }
+ 
+   protected static final Table FOO = new Table("foo", "1");
+   protected static final Table BAR = new Table("bar", "2");
+   protected static final Table BAZ = new Table("baz", "3");
+ 
+   protected final TestInstance instance = new TestInstance();
+   protected final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(instance);
+   protected final Map<String,String> servers = new HashMap<>(15);
+   protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers =
new TreeMap<>();
+   protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
+ 
+   {
+     servers.put("192.168.0.1", "r01s01");
+     servers.put("192.168.0.2", "r01s02");
+     servers.put("192.168.0.3", "r01s03");
+     servers.put("192.168.0.4", "r01s04");
+     servers.put("192.168.0.5", "r01s05");
+     servers.put("192.168.0.6", "r02s01");
+     servers.put("192.168.0.7", "r02s02");
+     servers.put("192.168.0.8", "r02s03");
+     servers.put("192.168.0.9", "r02s04");
+     servers.put("192.168.0.10", "r02s05");
+     servers.put("192.168.0.11", "r03s01");
+     servers.put("192.168.0.12", "r03s02");
+     servers.put("192.168.0.13", "r03s03");
+     servers.put("192.168.0.14", "r03s04");
+     servers.put("192.168.0.15", "r03s05");
+ 
+     allTabletServers.put(new TServerInstance("192.168.0.1:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.2:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.3:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.4:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.5:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.6:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.7:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.8:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.9:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.10:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.11:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.12:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.13:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
+     allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
+ 
+     tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>());
 -    tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("1"),
new Text("0")));
 -    tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("2"),
new Text("1")));
 -    tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("3"),
new Text("2")));
 -    tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("4"),
new Text("3")));
 -    tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("5"),
new Text("4")));
++    tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("1"), new
Text("0")));
++    tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("2"), new
Text("1")));
++    tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("3"), new
Text("2")));
++    tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("4"), new
Text("3")));
++    tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("5"), new
Text("4")));
+     tableExtents.put(BAR.getTableName(), new ArrayList<KeyExtent>());
 -    tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("11"),
new Text("10")));
 -    tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("12"),
new Text("11")));
 -    tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("13"),
new Text("12")));
 -    tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("14"),
new Text("13")));
 -    tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("15"),
new Text("14")));
++    tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("11"),
new Text("10")));
++    tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("12"),
new Text("11")));
++    tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("13"),
new Text("12")));
++    tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("14"),
new Text("13")));
++    tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("15"),
new Text("14")));
+     tableExtents.put(BAZ.getTableName(), new ArrayList<KeyExtent>());
 -    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("21"),
new Text("20")));
 -    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("22"),
new Text("21")));
 -    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("23"),
new Text("22")));
 -    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("24"),
new Text("23")));
 -    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("25"),
new Text("24")));
++    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("21"),
new Text("20")));
++    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("22"),
new Text("21")));
++    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("23"),
new Text("22")));
++    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("24"),
new Text("23")));
++    tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("25"),
new Text("24")));
+   }
+ 
+   protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
+     String tid = ke.getTableId().toString();
+     String host = tsi.host();
+     if (tid.equals("1")
+         && (host.equals("192.168.0.1") || host.equals("192.168.0.2") || host.equals("192.168.0.3")
|| host.equals("192.168.0.4") || host.equals("192.168.0.5"))) {
+       return true;
+     } else if (tid.equals("2")
+         && (host.equals("192.168.0.6") || host.equals("192.168.0.7") || host.equals("192.168.0.8")
|| host.equals("192.168.0.9") || host.equals("192.168.0.10"))) {
+       return true;
+     } else if (tid.equals("3")
+         && (host.equals("192.168.0.11") || host.equals("192.168.0.12") || host.equals("192.168.0.13")
|| host.equals("192.168.0.14") || host
+             .equals("192.168.0.15"))) {
+       return true;
+     } else {
+       return false;
+     }
+   }
+ 
+   @Override
+   protected TableOperations getTableOperations() {
+     return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
+       @Override
+       public Map<String,String> tableIdMap() {
+         HashMap<String,String> tables = new HashMap<>();
+         tables.put(FOO.getTableName(), FOO.getId());
+         tables.put(BAR.getTableName(), BAR.getId());
+         tables.put(BAZ.getTableName(), BAZ.getId());
+         return tables;
+       }
+     };
+   }
+ 
+   @Override
+   protected TabletBalancer getBalancerForTable(String table) {
+     return new DefaultLoadBalancer();
+   }
+ 
+   @Override
+   protected String getNameFromIp(String hostIp) throws UnknownHostException {
+     if (servers.containsKey(hostIp)) {
+       return servers.get(hostIp);
+     } else {
+       throw new UnknownHostException();
+     }
+   }
+ 
+   protected SortedMap<TServerInstance,TabletServerStatus> createCurrent(int numTservers)
{
+     String base = "192.168.0.";
+     TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
+     for (int i = 1; i <= numTservers; i++) {
+       current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus());
+     }
+     return current;
+   }
+ 
+ }


Mime
View raw message