accumulo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (ACCUMULO-4173) Balance table within a set of hosts
Date Wed, 30 Mar 2016 17:30:25 GMT

    [ https://issues.apache.org/jira/browse/ACCUMULO-4173?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15218400#comment-15218400
] 

ASF GitHub Bot commented on ACCUMULO-4173:
------------------------------------------

Github user brianloss commented on a diff in the pull request:

    https://github.com/apache/accumulo/pull/83#discussion_r57929241
  
    --- Diff: server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
---
    @@ -0,0 +1,314 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.accumulo.server.master.balancer;
    +
    +import java.net.InetAddress;
    +import java.net.UnknownHostException;
    +import java.util.ArrayList;
    +import java.util.Collections;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Map.Entry;
    +import java.util.Set;
    +import java.util.SortedMap;
    +import java.util.TreeMap;
    +import java.util.regex.Pattern;
    +
    +import org.apache.accumulo.core.client.admin.TableOperations;
    +import org.apache.accumulo.core.conf.AccumuloConfiguration;
    +import org.apache.accumulo.core.conf.Property;
    +import org.apache.accumulo.core.data.impl.KeyExtent;
    +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
    +import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
    +import org.apache.accumulo.server.conf.ServerConfiguration;
    +import org.apache.accumulo.server.master.state.TServerInstance;
    +import org.apache.accumulo.server.master.state.TabletMigration;
    +import org.apache.thrift.TException;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +/**
    + * This balancer will create pools of tablet servers by grouping tablet servers that
match a regex into the same pool and calling the balancer set on the table
    + * to balance within the set of matching tablet servers. <br>
    + * Regex properties for this balancer are specified as:<br>
    + * <b>table.custom.balancer.host.regex.&lt;tablename&gt;=&lt;regex&gt;</b><br>
    + * Periodically (default 5m) this balancer will check to see if a tablet server is hosting
tablets that it should not be according to the regex configuration.
    + * If this occurs then the offending tablets will be reassigned. This would cover the
case where the configuration is changed and the master is restarted while
    + * the tablet servers are up. To change the out of bounds check time period, set the
following property:<br>
    + * <b>table.custom.balancer.host.regex.oob.period=5m</b><br>
    + * Periodically (default 5m) this balancer will regroup the set of current tablet servers
into pools based on regexes applied to the tserver host names. This
    + * would cover the case of tservers dying or coming online. To change the host pool check
time period, set the following property: <br>
    + * <b>table.custom.balancer.host.regex.pool.check=5m</b><br>
    + * Regex matching can be based on either the host name (default) or host ip address.
To set this balancer to match the regular expressions to the tablet server
    + * IP address, then set the following property:<br>
    + * <b>table.custom.balancer.host.regex.is.ip=true</b>
    + *
    + */
    +public class HostRegexTableLoadBalancer extends TableLoadBalancer {
    +
    +  private static final Logger LOG = LoggerFactory.getLogger(HostRegexTableLoadBalancer.class);
    +  public static final String HOST_BALANCER_PREFIX = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ "balancer.host.regex.";
    +  public static final String HOST_BALANCER_OOB_CHECK = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ "balancer.host.regex.oob.period";
    +  private static final String HOST_BALANCER_OOB_DEFAULT = "5m";
    +  public static final String HOST_BALANCER_POOL_RECHECK_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ "balancer.host.regex.pool.check";
    +  private static final String HOST_BALANCER_POOL_RECHECK_DEFAULT = "5m";
    +  public static final String HOST_BALANCER_REGEX_USING_IPS = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ "balancer.host.regex.is.ip";
    +  protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
    +
    +  protected long oobCheckMillis = AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
    +  protected long poolRecheckMillis = AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_POOL_RECHECK_DEFAULT);
    +
    +  private Map<String,String> tableIdToTableName = null;
    +  private Map<String,Pattern> poolNameToRegexPattern = null;
    +  private long lastOOBCheck = System.currentTimeMillis();
    +  private long lastPoolRecheck = 0;
    +  private boolean isIpBasedRegex = false;
    +  private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools
= new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
    +
    +  /**
    +   * Group the set of current tservers by pool name. Tservers that don't match a regex
are put into a default ppol.
    +   *
    +   * @param current
    +   *          map of current tservers
    +   * @return current servers grouped by pool name, if not a match it is put into a default
pool.
    +   */
    +  protected synchronized Map<String,SortedMap<TServerInstance,TabletServerStatus>>
splitCurrentByRegex(SortedMap<TServerInstance,TabletServerStatus> current) {
    +    if ((System.currentTimeMillis() - lastPoolRecheck) > poolRecheckMillis) {
    +      Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools
= new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
    +      for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
    +        String tableName = getPoolNameForHost(e.getKey().host());
    +        if (!newPools.containsKey(tableName)) {
    +          newPools.put(tableName, new TreeMap<TServerInstance,TabletServerStatus>());
    +        }
    +        newPools.get(tableName).put(e.getKey(), e.getValue());
    +      }
    +      // Ensure that no host is in more than one pool
    +      // TODO: I'm not sure that I need to check for disjoint as the call to getPoolNameForHost
checks for more than one match
    +      boolean error = false;
    +      for (SortedMap<TServerInstance,TabletServerStatus> s1 : newPools.values())
{
    +        for (SortedMap<TServerInstance,TabletServerStatus> s2 : newPools.values())
{
    +          if (s1 == s2) {
    +            continue;
    +          }
    +          if (!Collections.disjoint(s1.keySet(), s2.keySet())) {
    +            LOG.error("Pools are not disjoint: {}, there is a problem with your regular
expressions. Putting all servers in default pool", newPools);
    +            error = true;
    +          }
    +        }
    +      }
    +      if (error) {
    +        // Put all servers into the default pool
    +        newPools.clear();
    +        newPools.put(DEFAULT_POOL, new TreeMap<TServerInstance,TabletServerStatus>());
    +        for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet())
{
    --- End diff --
    
    Nitpick, so ignore if you want, but you could replace this loop with newPools.putAll(current).
(Same comment as above about the comparator also applies.)


> Balance table within a set of hosts
> -----------------------------------
>
>                 Key: ACCUMULO-4173
>                 URL: https://issues.apache.org/jira/browse/ACCUMULO-4173
>             Project: Accumulo
>          Issue Type: Bug
>          Components: master
>            Reporter: Dave Marion
>            Assignee: Dave Marion
>              Labels: balancer
>             Fix For: 1.8.0
>
>         Attachments: ACCUMULO-4173-1.patch
>
>
> Create a table balancer that will provide a set of hosts for the table tablet balancer
to use.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message