accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [6/6] accumulo git commit: Merge branch '1.8'
Date Fri, 28 Jul 2017 00:28:18 GMT
Merge branch '1.8'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8c3c0783
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8c3c0783
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8c3c0783

Branch: refs/heads/master
Commit: 8c3c0783a1273b8237a4c358d8f0d39067f360d4
Parents: dddbba9 a205559
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Thu Jul 27 20:27:10 2017 -0400
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Thu Jul 27 20:27:10 2017 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/client/impl/ThriftScanner.java    | 1 -
 .../java/org/apache/accumulo/core/client/mock/MockTable.java   | 2 +-
 .../accumulo/core/client/summary/SummarizerConfiguration.java  | 2 +-
 .../accumulo/core/iterators/system/LocalityGroupIterator.java  | 4 ++--
 .../org/apache/accumulo/core/summary/SummaryCollection.java    | 1 -
 .../accumulo/core/iterators/system/ColumnFilterTest.java       | 6 +++---
 .../apache/accumulo/core/summary/SummaryCollectionTest.java    | 4 ++--
 .../server/master/balancer/HostRegexTableLoadBalancer.java     | 2 +-
 .../org/apache/accumulo/server/security/UserImpersonation.java | 1 -
 .../master/balancer/BaseHostRegexTableLoadBalancerTest.java    | 4 ++--
 .../src/main/java/org/apache/accumulo/master/Master.java       | 2 +-
 .../apache/accumulo/tserver/TabletServerResourceManager.java   | 2 +-
 .../apache/accumulo/tserver/LargestFirstMemoryManagerTest.java | 2 +-
 .../org/apache/accumulo/test/BalanceWithOfflineTableIT.java    | 1 -
 .../apache/accumulo/test/functional/FunctionalTestUtils.java   | 2 +-
 15 files changed, 16 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java
index ec98695,0000000..45fb7b1
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java
@@@ -1,285 -1,0 +1,285 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.summary;
 +
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.summary.SummarizerConfigurationUtil;
 +
 +import com.google.common.base.Preconditions;
 +import com.google.common.collect.ImmutableMap;
 +import com.google.common.hash.Hasher;
 +import com.google.common.hash.Hashing;
 +
 +/**
 + * This class encapsulates the configuration needed to instantiate a {@link Summarizer}.
It also provides methods and documentation for setting the table
 + * properties that configure a Summarizer.
 + *
 + * @since 2.0.0
 + */
 +public class SummarizerConfiguration {
 +
 +  private final String className;
 +  private final Map<String,String> options;
 +  private int hashCode = 0;
 +  private final String configId;
 +
 +  private SummarizerConfiguration(String className, String configId, Map<String,String>
options) {
 +    this.className = className;
 +    this.options = ImmutableMap.copyOf(options);
 +
 +    if (configId == null) {
 +      ArrayList<String> keys = new ArrayList<>(this.options.keySet());
 +      Collections.sort(keys);
 +      Hasher hasher = Hashing.murmur3_32().newHasher();
 +      hasher.putString(className);
 +      for (String key : keys) {
 +        hasher.putString(key);
 +        hasher.putString(options.get(key));
 +      }
 +
 +      this.configId = hasher.hash().toString();
 +    } else {
 +      this.configId = configId;
 +    }
 +  }
 +
 +  /**
 +   * @return the name of a class that implements @link {@link Summarizer}.
 +   */
 +  public String getClassName() {
 +    return className;
 +  }
 +
 +  /**
 +   * @return custom options for a {link @Summarizer}
 +   */
 +  public Map<String,String> getOptions() {
 +    return options;
 +  }
 +
 +  /**
 +   * The propertyId is used to when creating table properties for a summarizer. Its not
used for equality or hashCode for this class.
 +   */
 +  public String getPropertyId() {
 +    return configId;
 +  }
 +
 +  @Override
 +  public String toString() {
 +    return className + " " + configId + " " + options;
 +  }
 +
 +  /**
 +   * Compares the classname and options to determine equality.
 +   */
 +  @Override
 +  public boolean equals(Object o) {
 +    if (o instanceof SummarizerConfiguration) {
 +      SummarizerConfiguration osc = (SummarizerConfiguration) o;
 +      return className.equals(osc.className) && options.equals(osc.options);
 +    }
 +
 +    return false;
 +  }
 +
 +  /**
 +   * Hashes the classname and options to create a hashcode.
 +   */
 +  @Override
 +  public int hashCode() {
 +    if (hashCode == 0) {
 +      hashCode = 31 * options.hashCode() + className.hashCode();
 +    }
 +    return hashCode;
 +  }
 +
 +  /**
 +   * Converts this configuration to Accumulo per table properties. The returned map has
the following key values. The {@code <configId>} below is from
 +   * {@link #getPropertyId()}. The {@code <optionKey>} and {@code <optionValue>}
below are derived from the key values of {@link #getOptions()}.
 +   *
 +   * <pre>
 +   * {@code
 +   *   table.summarizer.<configId>=<classname>
 +   *   table.summarizer.<configId>.opt.<optionKey1>=<optionValue1>
 +   *   table.summarizer.<configId>.opt.<optionKey2>=<optionValue2>
 +   *      .
 +   *      .
 +   *      .
 +   *   table.summarizer.<configId>.opt.<optionKeyN>=<optionValueN>
 +   * }
 +   * </pre>
 +   */
 +  public Map<String,String> toTableProperties() {
 +    return SummarizerConfigurationUtil.toTablePropertiesMap(Collections.singletonList(this));
 +  }
 +
 +  /**
 +   * Encodes each configuration in the same way as {@link #toTableProperties()}.
 +   *
 +   * @throws IllegalArgumentException
 +   *           when there are duplicate values for {@link #getPropertyId()}
 +   */
 +  public static Map<String,String> toTableProperties(SummarizerConfiguration... configurations)
{
 +    return SummarizerConfigurationUtil.toTablePropertiesMap(Arrays.asList(configurations));
 +  }
 +
 +  /**
 +   * Encodes each configuration in the same way as {@link #toTableProperties()}.
 +   *
 +   * @throws IllegalArgumentException
 +   *           when there are duplicate values for {@link #getPropertyId()}
 +   */
 +  public static Map<String,String> toTableProperties(Collection<SummarizerConfiguration>
configurations) {
-     return SummarizerConfigurationUtil.toTablePropertiesMap(new ArrayList<SummarizerConfiguration>(configurations));
++    return SummarizerConfigurationUtil.toTablePropertiesMap(new ArrayList<>(configurations));
 +  }
 +
 +  /**
 +   * Decodes table properties with the prefix {@code table.summarizer} into {@link SummarizerConfiguration}
objects. Table properties with prefixes other than
 +   * {@code table.summarizer} are ignored.
 +   */
 +  public static Collection<SummarizerConfiguration> fromTableProperties(Map<String,String>
props) {
 +    return fromTableProperties(props.entrySet());
 +  }
 +
 +  /**
 +   * @see #fromTableProperties(Map)
 +   */
 +  public static Collection<SummarizerConfiguration> fromTableProperties(Iterable<Entry<String,String>>
props) {
 +    return SummarizerConfigurationUtil.getSummarizerConfigs(props);
 +  }
 +
 +  public static class Builder {
 +    private String className;
 +    private ImmutableMap.Builder<String,String> imBuilder;
 +    private String configId = null;
 +
 +    private Builder(String className) {
 +      this.className = className;
 +      this.imBuilder = ImmutableMap.builder();
 +    }
 +
 +    /**
 +     * Sets the id used when generating table properties. Setting this is optional. If not
set, an id is generated using hashing that will likely be unique.
 +     *
 +     * @param propId
 +     *          This id is used when converting a {@link SummarizerConfiguration} to table
properties. Since tables can have multiple summarizers, make sure its
 +     *          unique.
 +     *
 +     * @see SummarizerConfiguration#toTableProperties()
 +     */
 +    public Builder setPropertyId(String propId) {
 +      Preconditions.checkArgument(propId.matches("\\w+"), "Config Id %s is not alphanum",
propId);
 +      this.configId = propId;
 +      return this;
 +    }
 +
 +    /**
 +     * Adds an option that Summarizers can use when constructing Collectors and Combiners.
 +     *
 +     * @return this
 +     *
 +     * @see SummarizerConfiguration#getOptions()
 +     */
 +    public Builder addOption(String key, String value) {
 +      Preconditions.checkArgument(key.matches("\\w+"), "Option Id %s is not alphanum", key);
 +      imBuilder.put(key, value);
 +      return this;
 +    }
 +
 +    /**
 +     * Adds an option that Summarizers can use when constructing Collectors and Combiners.
 +     *
 +     * @return this
 +     *
 +     * @see SummarizerConfiguration#getOptions()
 +     */
 +    public Builder addOption(String key, long value) {
 +      return addOption(key, Long.toString(value));
 +    }
 +
 +    /**
 +     * Convenience method for adding multiple options. The following
 +     *
 +     * <pre>
 +     * {@code builder.addOptions("opt1","val1","opt2","val2","opt3","val3")}
 +     * </pre>
 +     *
 +     * <p>
 +     * is equivalent to
 +     *
 +     * <pre>
 +     * {@code
 +     *   builder.addOption("opt1","val1");
 +     *   builder.addOption("opt2","val2");
 +     *   builder.addOption("opt3","val3");
 +     * }
 +     * </pre>
 +     *
 +     * @param keyValuePairs
 +     *          This array must have an even and positive number of elements.
 +     * @return this
 +     * @see SummarizerConfiguration#getOptions()
 +     */
 +    public Builder addOptions(String... keyValuePairs) {
 +      Preconditions.checkArgument(keyValuePairs.length % 2 == 0 && keyValuePairs.length
> 0, "Require an even, positive number of arguments, got %s",
 +          keyValuePairs.length);
 +      for (int i = 0; i < keyValuePairs.length; i += 2) {
 +        addOption(keyValuePairs[i], keyValuePairs[i + 1]);
 +      }
 +      return this;
 +    }
 +
 +    /**
 +     * @param options
 +     *          Each entry in the map is passed to {@link #addOption(String, String)}
 +     * @return this
 +     *
 +     * @see SummarizerConfiguration#getOptions()
 +     */
 +    public Builder addOptions(Map<String,String> options) {
 +      options.entrySet().forEach(e -> addOption(e.getKey(), e.getValue()));
 +      return this;
 +    }
 +
 +    public SummarizerConfiguration build() {
 +      return new SummarizerConfiguration(className, configId, imBuilder.build());
 +    }
 +  }
 +
 +  /**
 +   * Call this method to initiate a chain of fluent method calls to a create an immutable
{@link SummarizerConfiguration}
 +   *
 +   * @param className
 +   *          The fully qualified name of a class that implements {@link Summarizer}.
 +   */
 +  public static Builder builder(String className) {
 +    return new Builder(className);
 +  }
 +
 +  /**
 +   * @see #builder(String)
 +   */
 +  public static Builder builder(Class<? extends Summarizer> clazz) {
 +    return new Builder(clazz.getName());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java
index cc688c9,0000000..cf216ed
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java
@@@ -1,188 -1,0 +1,187 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.accumulo.core.summary;
 +
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.client.summary.Summarizer;
 +import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
 +import org.apache.accumulo.core.client.summary.Summary;
 +import org.apache.accumulo.core.data.thrift.TSummaries;
 +import org.apache.accumulo.core.data.thrift.TSummarizerConfiguration;
 +import org.apache.accumulo.core.data.thrift.TSummary;
 +
 +import com.google.common.base.Preconditions;
 +
 +/**
 + * This class facilitates merging, storing, and serializing (to/from thrift) intermediate
summary information.
 + */
 +public class SummaryCollection {
 +
 +  private static class MergedSummary {
 +    Map<String,Long> summary;
 +    long filesContaining;
 +    long filesExceedingBoundry;
 +    long filesLarge;
 +
 +    public MergedSummary(FileSummary entry) {
 +      this.summary = entry.summary;
 +      this.filesContaining = 1;
 +      this.filesExceedingBoundry = entry.exceededBoundry ? 1 : 0;
 +      this.filesLarge = entry.exceededMaxSize ? 1 : 0;
 +    }
 +
 +    public MergedSummary(TSummary tSummary) {
 +      this.summary = new HashMap<>(tSummary.getSummary());
 +      this.filesContaining = tSummary.getFilesContaining();
 +      this.filesExceedingBoundry = tSummary.getFilesExceeding();
 +      this.filesLarge = tSummary.getFilesLarge();
 +    }
 +
 +    public void merge(MergedSummary other, SummarizerConfiguration config, SummarizerFactory
factory) {
 +
 +      if (summary == null && other.summary != null) {
 +        summary = new HashMap<>(other.summary);
 +      } else if (summary != null && other.summary != null) {
 +        Summarizer summarizer = factory.getSummarizer(config);
 +        summarizer.combiner(config).merge(summary, other.summary);
 +      }
 +
 +      filesContaining += other.filesContaining;
 +      filesExceedingBoundry += other.filesExceedingBoundry;
 +      filesLarge += other.filesLarge;
 +    }
 +
 +    public TSummary toThrift(SummarizerConfiguration key) {
 +      TSummarizerConfiguration tsumConf = SummarizerConfigurationUtil.toThrift(key);
 +      return new TSummary(summary, tsumConf, filesContaining, filesExceedingBoundry, filesLarge);
 +    }
 +
 +  }
 +
 +  private Map<SummarizerConfiguration,MergedSummary> mergedSummaries;
 +  private long totalFiles;
 +  private long deletedFiles;
 +
 +  public SummaryCollection() {
 +    mergedSummaries = new HashMap<>();
 +    totalFiles = 0;
 +  }
 +
 +  public SummaryCollection(TSummaries tsums) {
 +    mergedSummaries = new HashMap<>();
 +    for (TSummary tSummary : tsums.getSummaries()) {
 +      SummarizerConfiguration sconf = SummarizerConfigurationUtil.fromThrift(tSummary.getConfig());
 +      mergedSummaries.put(sconf, new MergedSummary(tSummary));
 +    }
 +
 +    totalFiles = tsums.getTotalFiles();
 +    deletedFiles = tsums.getDeletedFiles();
 +  }
 +
 +  SummaryCollection(Collection<FileSummary> initialEntries) {
 +    this(initialEntries, false);
 +  }
 +
 +  SummaryCollection(Collection<FileSummary> initialEntries, boolean deleted) {
 +    if (deleted) {
 +      Preconditions.checkArgument(initialEntries.size() == 0);
 +    }
 +    mergedSummaries = new HashMap<>();
 +    for (FileSummary entry : initialEntries) {
 +      mergedSummaries.put(entry.conf, new MergedSummary(entry));
 +    }
 +    totalFiles = 1;
 +    this.deletedFiles = deleted ? 1 : 0;
 +  }
 +
 +  static class FileSummary {
 +
 +    private SummarizerConfiguration conf;
 +    private Map<String,Long> summary;
 +    private boolean exceededBoundry;
 +    private boolean exceededMaxSize;
 +
 +    FileSummary(SummarizerConfiguration conf, Map<String,Long> summary, boolean exceededBoundry)
{
 +      this.conf = conf;
 +      this.summary = summary;
 +      this.exceededBoundry = exceededBoundry;
 +      this.exceededMaxSize = false;
 +    }
 +
 +    FileSummary(SummarizerConfiguration conf) {
 +      this.conf = conf;
 +      this.summary = new HashMap<>();
-       ;
 +      this.exceededBoundry = false;
 +      this.exceededMaxSize = true;
 +    }
 +  }
 +
 +  public void merge(SummaryCollection other, SummarizerFactory factory) {
 +    for (Entry<SummarizerConfiguration,MergedSummary> entry : other.mergedSummaries.entrySet())
{
 +      MergedSummary ms = mergedSummaries.get(entry.getKey());
 +      if (ms == null) {
 +        mergedSummaries.put(entry.getKey(), entry.getValue());
 +      } else {
 +        ms.merge(entry.getValue(), entry.getKey(), factory);
 +      }
 +    }
 +
 +    this.totalFiles += other.totalFiles;
 +    this.deletedFiles += other.deletedFiles;
 +  }
 +
 +  public static SummaryCollection merge(SummaryCollection sc1, SummaryCollection sc2, SummarizerFactory
factory) {
 +    SummaryCollection ret = new SummaryCollection();
 +    ret.merge(sc1, factory);
 +    ret.merge(sc2, factory);
 +    return ret;
 +  }
 +
 +  public List<Summary> getSummaries() {
 +    ArrayList<Summary> ret = new ArrayList<>(mergedSummaries.size());
 +
 +    for (Entry<SummarizerConfiguration,MergedSummary> entry : mergedSummaries.entrySet())
{
 +      SummarizerConfiguration config = entry.getKey();
 +      MergedSummary ms = entry.getValue();
 +
 +      ret.add(new Summary(ms.summary, config, totalFiles, (totalFiles - deletedFiles) -
ms.filesContaining, ms.filesExceedingBoundry, ms.filesLarge,
 +          deletedFiles));
 +    }
 +
 +    return ret;
 +  }
 +
 +  public long getTotalFiles() {
 +    return totalFiles;
 +  }
 +
 +  public TSummaries toThrift() {
 +    List<TSummary> summaries = new ArrayList<>(mergedSummaries.size());
 +    for (Entry<SummarizerConfiguration,MergedSummary> entry : mergedSummaries.entrySet())
{
 +      summaries.add(entry.getValue().toThrift(entry.getKey()));
 +    }
 +
 +    return new TSummaries(true, -1l, totalFiles, deletedFiles, summaries);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java
index ab9c975,4b03826..dd776af
--- a/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java
@@@ -46,9 -47,9 +46,9 @@@ public class ColumnFilterTest extends T
    }
  
    public void test1() {
-     TreeMap<Key,Value> data = new TreeMap<Key,Value>();
+     TreeMap<Key,Value> data = new TreeMap<>();
 -    data.put(newKey("r1", "cf1", "cq1"), new Value());
 -    data.put(newKey("r1", "cf2", "cq1"), new Value());
 +    data.put(newKey("r1", "cf1", "cq1"), new Value(""));
 +    data.put(newKey("r1", "cf2", "cq1"), new Value(""));
  
      HashSet<Column> columns = new HashSet<>();
      columns.add(newColumn("cf1"));
@@@ -61,10 -62,10 +61,10 @@@
  
    public void test2() throws Exception {
  
-     TreeMap<Key,Value> data = new TreeMap<Key,Value>();
+     TreeMap<Key,Value> data = new TreeMap<>();
 -    data.put(newKey("r1", "cf1", "cq1"), new Value());
 -    data.put(newKey("r1", "cf2", "cq1"), new Value());
 -    data.put(newKey("r1", "cf2", "cq2"), new Value());
 +    data.put(newKey("r1", "cf1", "cq1"), new Value(""));
 +    data.put(newKey("r1", "cf2", "cq1"), new Value(""));
 +    data.put(newKey("r1", "cf2", "cq2"), new Value(""));
  
      HashSet<Column> columns = new HashSet<>();
  
@@@ -85,10 -86,10 +85,10 @@@
  
    public void test3() throws Exception {
  
-     TreeMap<Key,Value> data = new TreeMap<Key,Value>();
+     TreeMap<Key,Value> data = new TreeMap<>();
 -    data.put(newKey("r1", "cf1", "cq1"), new Value());
 -    data.put(newKey("r1", "cf2", "cq1"), new Value());
 -    data.put(newKey("r1", "cf2", "cq2"), new Value());
 +    data.put(newKey("r1", "cf1", "cq1"), new Value(""));
 +    data.put(newKey("r1", "cf2", "cq1"), new Value(""));
 +    data.put(newKey("r1", "cf2", "cq2"), new Value(""));
  
      HashSet<Column> columns = new HashSet<>();
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java
index 95702d3,0000000..ae1dab5
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java
@@@ -1,72 -1,0 +1,72 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.accumulo.core.summary;
 +
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
 +import org.apache.accumulo.core.client.summary.Summary;
 +import org.apache.accumulo.core.client.summary.Summary.FileStatistics;
 +import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer;
 +import org.apache.accumulo.core.summary.SummaryCollection.FileSummary;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +public class SummaryCollectionTest {
 +  @Test
 +  public void testDeleted() {
 +    SummarizerConfiguration conf = SummarizerConfiguration.builder(FamilySummarizer.class).build();
 +
-     HashMap<String,Long> stats = new HashMap<String,Long>();
++    HashMap<String,Long> stats = new HashMap<>();
 +    stats.put("c:foo", 9L);
 +    FileSummary fs1 = new FileSummary(conf, stats, false);
 +    SummaryCollection sc1 = new SummaryCollection(Collections.singleton(fs1));
 +
-     stats = new HashMap<String,Long>();
++    stats = new HashMap<>();
 +    stats.put("c:foo", 5L);
 +    stats.put("c:bar", 3L);
 +    FileSummary fs2 = new FileSummary(conf, stats, true);
 +    SummaryCollection sc2 = new SummaryCollection(Collections.singleton(fs2));
 +
 +    SummaryCollection sc3 = new SummaryCollection(Collections.emptyList());
 +
 +    SummaryCollection sc4 = new SummaryCollection(Collections.emptyList(), true);
 +
 +    SummarizerFactory factory = new SummarizerFactory();
 +    SummaryCollection mergeSc = new SummaryCollection();
 +    for (SummaryCollection sc : Arrays.asList(sc1, sc2, sc3, sc4, sc4)) {
 +      mergeSc.merge(sc, factory);
 +    }
 +
 +    for (SummaryCollection sc : Arrays.asList(mergeSc, new SummaryCollection(mergeSc.toThrift())))
{
 +      List<Summary> summaries = sc.getSummaries();
 +      Assert.assertEquals(1, summaries.size());
 +      Summary summary = summaries.get(0);
 +      FileStatistics filestats = summary.getFileStatistics();
 +      Assert.assertEquals(5, filestats.getTotal());
 +      Assert.assertEquals(1, filestats.getExtra());
 +      Assert.assertEquals(0, filestats.getLarge());
 +      Assert.assertEquals(1, filestats.getMissing());
 +      Assert.assertEquals(2, filestats.getDeleted());
 +      Assert.assertEquals(4, filestats.getInaccurate());
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
index 4606c04,44cf76b..e6791f4
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
@@@ -338,10 -340,10 +338,10 @@@ public abstract class BaseHostRegexTabl
      TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
      for (int i = 1; i <= numTservers; i++) {
        TabletServerStatus status = new TabletServerStatus();
-       Map<String,TableInfo> tableMap = new HashMap<String,TableInfo>();
+       Map<String,TableInfo> tableMap = new HashMap<>();
 -      tableMap.put(FOO.getId(), new TableInfo());
 -      tableMap.put(BAR.getId(), new TableInfo());
 -      tableMap.put(BAZ.getId(), new TableInfo());
 +      tableMap.put(FOO.getId().canonicalID(), new TableInfo());
 +      tableMap.put(BAR.getId().canonicalID(), new TableInfo());
 +      tableMap.put(BAZ.getId().canonicalID(), new TableInfo());
        status.setTableMap(tableMap);
        current.put(new TServerInstance(base + i + ":9997", 1), status);
      }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index 7397c06,d8ae94c..cc498fb
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@@ -146,16 -139,8 +146,16 @@@ public class TabletServerResourceManage
      return createEs(max, name, new LinkedBlockingQueue<Runnable>());
    }
  
 +  private ExecutorService createIdlingEs(Property max, String name, long timeout, TimeUnit
timeUnit) {
-     LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
++    LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
 +    int maxThreads = conf.getSystemConfiguration().getCount(max);
 +    ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, timeout, timeUnit,
queue, new NamingThreadFactory(name));
 +    tp.allowCoreThreadTimeOut(true);
 +    return addEs(max, name, tp);
 +  }
 +
    private ExecutorService createEs(Property max, String name, BlockingQueue<Runnable>
queue) {
 -    int maxThreads = conf.getConfiguration().getCount(max);
 +    int maxThreads = conf.getSystemConfiguration().getCount(max);
      ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS,
queue, new NamingThreadFactory(name));
      return addEs(max, name, tp);
    }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
index 27045c3,82ec8ec..42b70a5
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
@@@ -172,7 -176,12 +172,7 @@@ public class LargestFirstMemoryManagerT
    @Test
    public void testDeletedTable() throws Exception {
      final String deletedTableId = "1";
-     Function<Table.ID,Boolean> existenceCheck = tableId -> !deletedTableId.equals(tableId);
 -    Function<String,Boolean> existenceCheck = new Function<String,Boolean>()
{
 -      @Override
 -      public Boolean apply(String tableId) {
 -        return !deletedTableId.equals(tableId);
 -      }
 -    };
++    Function<Table.ID,Boolean> existenceCheck = tableId -> !deletedTableId.contentEquals(tableId.canonicalID());
      LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
      ServerConfiguration config = new ServerConfiguration() {
        ServerConfigurationFactory delegate = new ServerConfigurationFactory(inst);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
----------------------------------------------------------------------


Mime
View raw message