accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject accumulo git commit: WIP
Date Fri, 13 Nov 2015 04:32:22 GMT
Repository: accumulo
Updated Branches:
  refs/heads/ACCUMULO-2883 [created] 1ae1b34f3


WIP


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1ae1b34f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1ae1b34f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1ae1b34f

Branch: refs/heads/ACCUMULO-2883
Commit: 1ae1b34f316ecd2e5b98e513a5eb3415f6365eb0
Parents: 0e136c1
Author: Keith Turner <keith@deenlo.com>
Authored: Thu Nov 12 23:31:22 2015 -0500
Committer: Keith Turner <keith@deenlo.com>
Committed: Thu Nov 12 23:31:22 2015 -0500

----------------------------------------------------------------------
 .../core/client/admin/TableOperations.java      |  14 ++
 .../core/client/impl/TableOperationsImpl.java   |  49 ++++++
 .../core/client/mock/MockTableOperations.java   |   7 +
 .../client/impl/TableOperationsHelperTest.java  |   7 +
 .../org/apache/accumulo/test/LocatorIT.java     | 166 +++++++++++++++++++
 5 files changed, 243 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ae1b34f/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index fa6fef4..0041e62 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TabletId;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.hadoop.io.Text;
@@ -231,6 +232,19 @@ public interface TableOperations {
   Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException,
AccumuloSecurityException, AccumuloException;
 
   /**
+   * Locates the tablet servers and tablets that would service a collections of ranges. If
a range covers multiple tablets, it will occur multiple times in the
+   * returned map.
+   *
+   * @param ranges
+   *          The input ranges that should be mapped to tablet servers and tablets.
+   * @return A map of tablet servers locations to a map of tablet locations to a list of
ranges.
+   *
+   * @since 1.8.0
+   */
+  Map<String,Map<TabletId,List<Range>>> locate(String tableName, Collection<Range>
ranges) throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException;
+
+  /**
    * Finds the max row within a given range. To find the max row in a table, pass null for
start and end row.
    *
    * @param auths

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ae1b34f/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 8434f2f..8071e3a 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@ -83,8 +83,10 @@ import org.apache.accumulo.core.constraints.Constraint;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TabletId;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.data.impl.TabletIdImpl;
 import org.apache.accumulo.core.iterators.IteratorUtil;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
@@ -120,6 +122,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 import com.google.common.net.HostAndPort;
 
 public class TableOperationsImpl extends TableOperationsHelper {
@@ -1513,4 +1516,50 @@ public class TableOperationsImpl extends TableOperationsHelper {
     return sci.toSamplerConfiguration();
   }
 
+  @Override
+  public Map<String,Map<TabletId,List<Range>>> locate(String tableName,
Collection<Range> ranges) throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException {
+    Preconditions.checkNotNull(tableName, "tableName must be non null");
+    Preconditions.checkNotNull(ranges, "ranges must be non null");
+
+    String tableId = Tables.getTableId(context.getInstance(), tableName);
+    TabletLocator locator = TabletLocator.getLocator(context, new Text(tableId));
+
+    List<Range> rangeList = null;
+    if (ranges instanceof List) {
+      rangeList = (List<Range>) ranges;
+    } else {
+      rangeList = new ArrayList<>(ranges);
+    }
+
+    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+
+    locator.invalidateCache();
+
+    while (!locator.binRanges(context, rangeList, binnedRanges).isEmpty()) {
+
+      if (!Tables.exists(context.getInstance(), tableId))
+        throw new TableDeletedException(tableId);
+      if (Tables.getTableState(context.getInstance(), tableId) == TableState.OFFLINE)
+        throw new TableOfflineException(context.getInstance(), tableId);
+
+      binnedRanges.clear();
+
+      // TODO sleep/use Retry
+
+      locator.invalidateCache();
+    }
+
+    Map<String,Map<TabletId,List<Range>>> ret = new HashMap<>();
+
+    for (Entry<String,Map<KeyExtent,List<Range>>> entry : binnedRanges.entrySet())
{
+      Map<TabletId,List<Range>> tabletMap = new HashMap();
+      ret.put(entry.getKey(), tabletMap);
+      for (Entry<KeyExtent,List<Range>> entry2 : entry.getValue().entrySet())
{
+        tabletMap.put(new TabletIdImpl(entry2.getKey()), entry2.getValue());
+      }
+    }
+
+    return ret;
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ae1b34f/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
index 7ca5766..6c885ce 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
@@ -48,6 +48,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TabletId;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.file.FileOperations;
 import org.apache.accumulo.core.file.FileSKVIterator;
@@ -497,4 +498,10 @@ class MockTableOperations extends TableOperationsHelper {
   public SamplerConfiguration getSamplerConfiguration(String tableName) throws TableNotFoundException,
AccumuloException, AccumuloSecurityException {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public Map<String,Map<TabletId,List<Range>>> locate(String tableName,
Collection<Range> ranges) throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ae1b34f/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java
b/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java
index 7bf9eb1..405d57b 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/impl/TableOperationsHelperTest.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration;
 import org.apache.accumulo.core.client.admin.SamplerConfiguration;
 import org.apache.accumulo.core.client.admin.TimeType;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TabletId;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.hadoop.io.Text;
@@ -243,6 +244,12 @@ public class TableOperationsHelperTest {
     public SamplerConfiguration getSamplerConfiguration(String tableName) throws TableNotFoundException,
AccumuloException, AccumuloSecurityException {
       throw new UnsupportedOperationException();
     }
+
+    @Override
+    public Map<String,Map<TabletId,List<Range>>> locate(String tableName,
Collection<Range> ranges) throws AccumuloException, AccumuloSecurityException,
+        TableNotFoundException {
+      throw new UnsupportedOperationException();
+    }
   }
 
   protected TableOperationsHelper getHelper() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ae1b34f/test/src/main/java/org/apache/accumulo/test/LocatorIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java
new file mode 100644
index 0000000..966eb58
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LocatorIT extends AccumuloClusterHarness {
+
+  private static class Expected {
+    private String prev;
+    private String end;
+    private Range range;
+
+    private Expected(String prev, String end, Range r) {
+      this.prev = prev == null ? "" : prev;
+      this.end = end == null ? "" : end;
+      this.range = r;
+    }
+
+    public Expected(TabletId tid, Range range2) {
+      prev = tid.getPrevEndRow() == null ? "" : tid.getPrevEndRow().toString();
+      end = tid.getEndRow() == null ? "" : tid.getEndRow().toString();
+      range = range2;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o instanceof Expected) {
+        Expected oexp = (Expected) o;
+        return prev.equals(oexp.prev) && end.equals(oexp.end) && range.equals(oexp.range);
+      }
+      return false;
+    }
+
+    @Override
+    public int hashCode() {
+      return prev.hashCode() + 31 * end.hashCode() + 19 * range.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return "[" + end + "] [" + prev + "] " + range;
+    }
+  }
+
+  private static Expected ne(String prev, String end, Range r) {
+    return new Expected(prev, end, r);
+  }
+
+  private void assertContains(Map<String,Map<TabletId,List<Range>>> br,
HashSet<String> tservers, Expected... expected) {
+    ArrayList<Expected> el = new ArrayList<>(Arrays.asList(expected));
+
+    Text tableId = null;
+
+    for (Entry<String,Map<TabletId,List<Range>>> entry : br.entrySet())
{
+      Assert.assertTrue("tserver " + entry.getKey() + " not found in " + tservers, tservers.contains(entry.getKey()));
+
+      for (Entry<TabletId,List<Range>> entry2 : entry.getValue().entrySet())
{
+        TabletId tid = entry2.getKey();
+
+        Assert.assertNotNull(tid.getTableId());
+
+        if (tableId == null) {
+          tableId = tid.getTableId();
+        } else {
+          Assert.assertEquals(tableId, tid.getTableId());
+        }
+
+        for (Range range : entry2.getValue()) {
+          Assert.assertTrue(el.size() > 0);
+
+          Iterator<Expected> it = el.iterator();
+
+          boolean found = false;
+
+          while (it.hasNext()) {
+            Expected exp = it.next();
+            if (exp.equals(new Expected(tid, range))) {
+              it.remove();
+              found = true;
+              break;
+            }
+          }
+
+          Assert.assertTrue("Did not find " + tid + " " + range, found);
+        }
+      }
+    }
+
+    Assert.assertTrue(el.toString(), el.size() == 0);
+
+  }
+
+  @Test
+  public void testBasic() throws Exception {
+    Connector conn = getConnector();
+    String tableName = getUniqueNames(1)[0];
+
+    conn.tableOperations().create(tableName);
+
+    Range r1 = new Range("m");
+    Range r2 = new Range("o", "x");
+
+    ArrayList<Range> ranges = new ArrayList<>();
+
+    Assert.assertEquals(0, conn.tableOperations().locate(tableName, ranges).size());
+
+    HashSet<String> tservers = new HashSet<>(conn.instanceOperations().getTabletServers());
+
+    ranges.add(r1);
+    Map<String,Map<TabletId,List<Range>>> ret = conn.tableOperations().locate(tableName,
ranges);
+    assertContains(ret, tservers, ne(null, null, r1));
+
+    ranges.add(r2);
+    ret = conn.tableOperations().locate(tableName, ranges);
+    assertContains(ret, tservers, ne(null, null, r1), ne(null, null, r2));
+
+    TreeSet<Text> splits = new TreeSet<Text>();
+    splits.add(new Text("r"));
+    conn.tableOperations().addSplits(tableName, splits);
+
+    ret = conn.tableOperations().locate(tableName, ranges);
+    assertContains(ret, tservers, ne(null, "r", r1), ne(null, "r", r2), ne("r", null, r2));
+
+    conn.tableOperations().offline(tableName, true);
+
+    try {
+      conn.tableOperations().locate(tableName, ranges);
+      Assert.fail();
+    } catch (TableOfflineException e) {
+
+    }
+
+  }
+}


Mime
View raw message