accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch main updated: Make Ample only return exact matches for KeyExtents (#2189)
Date Mon, 19 Jul 2021 15:53:16 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 548b037  Make Ample only return exact matches for KeyExtents (#2189)
548b037 is described below

commit 548b037c5917dc1e693733dc3d89599369d79103
Author: Dom G <47725857+DomGarguilo@users.noreply.github.com>
AuthorDate: Mon Jul 19 11:53:05 2021 -0400

    Make Ample only return exact matches for KeyExtents (#2189)
---
 .../core/metadata/schema/TabletsMetadata.java      | 69 +++++++++++++++-------
 .../accumulo/server/util/MetadataTableUtil.java    |  5 +-
 .../org/apache/accumulo/compactor/Compactor.java   |  3 +-
 .../org/apache/accumulo/tserver/TabletServer.java  |  5 ++
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  5 +-
 5 files changed, 58 insertions(+), 29 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 042786c..8ffa08e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
@@ -87,8 +88,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
 
   public static class Builder implements TableRangeOptions, TableOptions, RangeOptions, Options
{
 
-    private List<Text> families = new ArrayList<>();
-    private List<ColumnFQ> qualifiers = new ArrayList<>();
+    private final List<Text> families = new ArrayList<>();
+    private final List<ColumnFQ> qualifiers = new ArrayList<>();
+    private Set<KeyExtent> extentsToFetch = null;
     private Ample.DataLevel level;
     private String table;
     private Range range;
@@ -98,8 +100,8 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
     private boolean saveKeyValues;
     private TableId tableId;
     private ReadConsistency readConsistency = ReadConsistency.IMMEDIATE;
-    private AccumuloClient _client;
-    private Collection<KeyExtent> extents;
+    private final AccumuloClient _client;
+    private Collection<KeyExtent> extents = null;
 
     Builder(AccumuloClient client) {
       this._client = client;
@@ -132,19 +134,33 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
         var ranges = extents.stream().map(KeyExtent::toMetaRange).collect(toList());
         scanner.setRanges(ranges);
 
+        boolean extentsPresent = extentsToFetch != null;
+
+        if (!fetchedCols.isEmpty() && extentsPresent)
+          fetch(ColumnType.PREV_ROW);
+
         configureColumns(scanner);
         IteratorSetting iterSetting = new IteratorSetting(100, WholeRowIterator.class);
         scanner.addScanIterator(iterSetting);
 
-        Iterable<TabletMetadata> tmi = () -> Iterators.transform(scanner.iterator(),
entry -> {
-          try {
-            return TabletMetadata.convertRow(
-                WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(),
-                fetchedCols, saveKeyValues);
-          } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        Iterable<TabletMetadata> tmi = () -> {
+          Iterator<TabletMetadata> iter = Iterators.transform(scanner.iterator(), entry
-> {
+            try {
+              return TabletMetadata.convertRow(WholeRowIterator
+                  .decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(), fetchedCols,
+                  saveKeyValues);
+            } catch (IOException e) {
+              throw new UncheckedIOException(e);
+            }
+          });
+
+          if (extentsPresent) {
+            return Iterators.filter(iter,
+                tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent()));
+          } else {
+            return iter;
           }
-        });
+        };
 
         return new TabletsMetadata(scanner, tmi);
       } catch (TableNotFoundException e) {
@@ -161,7 +177,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
             new IsolatedScanner(client.createScanner(resolvedTable, Authorizations.EMPTY));
         scanner.setRange(range);
 
-        if (checkConsistency && !fetchedCols.contains(ColumnType.PREV_ROW)) {
+        boolean extentsPresent = extentsToFetch != null;
+
+        if (!fetchedCols.isEmpty() && (checkConsistency || extentsPresent)) {
           fetch(ColumnType.PREV_ROW);
         }
 
@@ -172,8 +190,14 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
           synchronized (scanner) {
             scanner.setRange(r);
             RowIterator rowIter = new RowIterator(scanner);
-            return Iterators.transform(rowIter,
+            Iterator<TabletMetadata> iter = Iterators.transform(rowIter,
                 ri -> TabletMetadata.convertRow(ri, fetchedCols, saveKeyValues));
+            if (extentsPresent) {
+              return Iterators.filter(iter,
+                  tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent()));
+            } else {
+              return iter;
+            }
           }
         };
 
@@ -291,6 +315,7 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
     public Options forTablet(KeyExtent extent) {
       forTable(extent.tableId());
       this.range = new Range(extent.toMetaRow());
+      this.extentsToFetch = Set.of(extent);
       return this;
     }
 
@@ -304,6 +329,7 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
 
       this.level = DataLevel.USER;
       this.extents = extents;
+      this.extentsToFetch = Set.copyOf(extents);
       return this;
     }
 
@@ -376,14 +402,13 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
 
     /**
      * Get the tablet metadata for this extents end row. This should only ever return a single
-     * tablet. No checking is done for prev row, so it could differ.
+     * tablet where the end row and prev end row exactly match the given extent.
      */
     Options forTablet(KeyExtent extent);
 
     /**
-     * Get the tablet metadata for the given extents. This will find tablets based on end
row, so
-     * it's possible the prev rows could differ for the tablets returned. If this matters,
then it
-     * must be checked.
+     * Get the tablet metadata for the given extents. This will only return tablets where
the end
+     * row and prev end row exactly match the given extents.
      */
     Options forTablets(Collection<KeyExtent> extents);
 
@@ -431,8 +456,8 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
   private static class TabletMetadataIterator implements Iterator<TabletMetadata> {
 
     private boolean sawLast = false;
-    private Iterator<TabletMetadata> iter;
-    private Text endRow;
+    private final Iterator<TabletMetadata> iter;
+    private final Text endRow;
 
     TabletMetadataIterator(Iterator<TabletMetadata> source, Text endRow) {
       this.iter = source;
@@ -487,9 +512,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>,
AutoCloseable
         .convertToTabletMetadata();
   }
 
-  private ScannerBase scanner;
+  private final ScannerBase scanner;
 
-  private Iterable<TabletMetadata> tablets;
+  private final Iterable<TabletMetadata> tablets;
 
   private TabletsMetadata(TabletMetadata tm) {
     this.scanner = null;
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 709cc7f..591cf5a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -418,8 +418,9 @@ public class MetadataTableUtil {
 
     TabletMetadata tablet = context.getAmple().readTablet(extent, FILES, LOGS, PREV_ROW,
DIR);
 
-    if (!tablet.getExtent().equals(extent))
-      throw new RuntimeException("Unexpected extent " + tablet.getExtent() + " expected "
+ extent);
+    if (tablet == null) {
+      throw new RuntimeException("Tablet " + extent + " not found in metadata");
+    }
 
     result.addAll(tablet.getLogs());
 
diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 41865f0..b63e1bd 100644
--- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -190,8 +190,7 @@ public class Compactor extends AbstractServer implements CompactorService.Iface
 
         TabletMetadata tabletMeta =
             getContext().getAmple().readTablet(extent, ColumnType.ECOMP, ColumnType.PREV_ROW);
-        if (tabletMeta == null || !tabletMeta.getExtent().equals(extent)
-            || !tabletMeta.getExternalCompactions().containsKey(ecid)) {
+        if (tabletMeta == null || !tabletMeta.getExternalCompactions().containsKey(ecid))
{
           // table was deleted OR tablet was split or merged OR tablet no longer thinks compaction
           // is running for some reason
           LOG.info("Cancelling compaction {} that no longer has a metadata entry at {}",
ecid,
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index c4bb1eb..1341838 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -926,6 +926,11 @@ public class TabletServer extends AbstractServer {
   static boolean checkTabletMetadata(KeyExtent extent, TServerInstance instance,
       TabletMetadata meta) throws AccumuloException {
 
+    if (meta == null) {
+      log.info("Not loading tablet {}, its metadata was not found.", extent);
+      return false;
+    }
+
     if (!meta.sawPrevEndRow()) {
       throw new AccumuloException("Metadata entry does not have prev row (" + meta.getTableId()
           + " " + meta.getEndRow() + ")");
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 522efff..ad93e70 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1361,9 +1361,8 @@ public class Tablet {
       var tabletMeta = context.getAmple().readTablet(extent, ColumnType.FILES, ColumnType.LOGS,
           ColumnType.ECOMP, ColumnType.PREV_ROW);
 
-      if (!tabletMeta.getExtent().equals(extent)) {
-        String msg = "Closed tablet " + extent + " does not match extent in metadata table
"
-            + tabletMeta.getExtent();
+      if (tabletMeta == null) {
+        String msg = "Closed tablet " + extent + " not found in metadata";
         log.error(msg);
         throw new RuntimeException(msg);
       }

Mime
View raw message