From commits-return-25081-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Mon Jul 19 15:53:17 2021 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mxout1-ec2-va.apache.org (mxout1-ec2-va.apache.org [3.227.148.255]) by mx-eu-01.ponee.io (Postfix) with ESMTPS id AD91D180663 for ; Mon, 19 Jul 2021 17:53:17 +0200 (CEST) Received: from mail.apache.org (mailroute1-lw-us.apache.org [207.244.88.153]) by mxout1-ec2-va.apache.org (ASF Mail Server at mxout1-ec2-va.apache.org) with SMTP id EA3E44018D for ; Mon, 19 Jul 2021 15:53:16 +0000 (UTC) Received: (qmail 88683 invoked by uid 500); 19 Jul 2021 15:53:16 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 88674 invoked by uid 99); 19 Jul 2021 15:53:16 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 19 Jul 2021 15:53:16 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 74E0F81EE9; Mon, 19 Jul 2021 15:53:16 +0000 (UTC) Date: Mon, 19 Jul 2021 15:53:16 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch main updated: Make Ample only return exact matches for KeyExtents (#2189) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <162670999605.22543.11221699043785834755@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Oldrev: f5df4f9681736d35e14b92c58033a1cf062b3d4e X-Git-Newrev: 548b037c5917dc1e693733dc3d89599369d79103 X-Git-Rev: 548b037c5917dc1e693733dc3d89599369d79103 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 548b037 Make Ample only return exact matches for KeyExtents (#2189) 548b037 is described below commit 548b037c5917dc1e693733dc3d89599369d79103 Author: Dom G <47725857+DomGarguilo@users.noreply.github.com> AuthorDate: Mon Jul 19 11:53:05 2021 -0400 Make Ample only return exact matches for KeyExtents (#2189) --- .../core/metadata/schema/TabletsMetadata.java | 69 +++++++++++++++------- .../accumulo/server/util/MetadataTableUtil.java | 5 +- .../org/apache/accumulo/compactor/Compactor.java | 3 +- .../org/apache/accumulo/tserver/TabletServer.java | 5 ++ .../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +- 5 files changed, 58 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java index 042786c..8ffa08e 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -87,8 +88,9 @@ public class TabletsMetadata implements Iterable, AutoCloseable public static class Builder implements TableRangeOptions, TableOptions, RangeOptions, Options { - private List families = new ArrayList<>(); - private List qualifiers = new ArrayList<>(); + private final List families = new ArrayList<>(); + private final List qualifiers = new ArrayList<>(); + private Set extentsToFetch = null; private Ample.DataLevel level; private String table; private Range range; @@ -98,8 +100,8 @@ public class TabletsMetadata implements Iterable, AutoCloseable private boolean saveKeyValues; private TableId tableId; private ReadConsistency readConsistency = ReadConsistency.IMMEDIATE; - private AccumuloClient _client; - private Collection extents; + private final AccumuloClient _client; + private Collection extents = null; Builder(AccumuloClient client) { this._client = client; @@ -132,19 +134,33 @@ public class TabletsMetadata implements Iterable, AutoCloseable var ranges = extents.stream().map(KeyExtent::toMetaRange).collect(toList()); scanner.setRanges(ranges); + boolean extentsPresent = extentsToFetch != null; + + if (!fetchedCols.isEmpty() && extentsPresent) + fetch(ColumnType.PREV_ROW); + configureColumns(scanner); IteratorSetting iterSetting = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(iterSetting); - Iterable tmi = () -> Iterators.transform(scanner.iterator(), entry -> { - try { - return TabletMetadata.convertRow( - WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(), - fetchedCols, saveKeyValues); - } catch (IOException e) { - throw new UncheckedIOException(e); + Iterable tmi = () -> { + Iterator iter = Iterators.transform(scanner.iterator(), entry -> { + try { + return TabletMetadata.convertRow(WholeRowIterator + .decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(), fetchedCols, + saveKeyValues); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + if (extentsPresent) { + return Iterators.filter(iter, + tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent())); + } else { + return iter; } - }); + }; return new TabletsMetadata(scanner, tmi); } catch (TableNotFoundException e) { @@ -161,7 +177,9 @@ public class TabletsMetadata implements Iterable, AutoCloseable new IsolatedScanner(client.createScanner(resolvedTable, Authorizations.EMPTY)); scanner.setRange(range); - if (checkConsistency && !fetchedCols.contains(ColumnType.PREV_ROW)) { + boolean extentsPresent = extentsToFetch != null; + + if (!fetchedCols.isEmpty() && (checkConsistency || extentsPresent)) { fetch(ColumnType.PREV_ROW); } @@ -172,8 +190,14 @@ public class TabletsMetadata implements Iterable, AutoCloseable synchronized (scanner) { scanner.setRange(r); RowIterator rowIter = new RowIterator(scanner); - return Iterators.transform(rowIter, + Iterator iter = Iterators.transform(rowIter, ri -> TabletMetadata.convertRow(ri, fetchedCols, saveKeyValues)); + if (extentsPresent) { + return Iterators.filter(iter, + tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent())); + } else { + return iter; + } } }; @@ -291,6 +315,7 @@ public class TabletsMetadata implements Iterable, AutoCloseable public Options forTablet(KeyExtent extent) { forTable(extent.tableId()); this.range = new Range(extent.toMetaRow()); + this.extentsToFetch = Set.of(extent); return this; } @@ -304,6 +329,7 @@ public class TabletsMetadata implements Iterable, AutoCloseable this.level = DataLevel.USER; this.extents = extents; + this.extentsToFetch = Set.copyOf(extents); return this; } @@ -376,14 +402,13 @@ public class TabletsMetadata implements Iterable, AutoCloseable /** * Get the tablet metadata for this extents end row. This should only ever return a single - * tablet. No checking is done for prev row, so it could differ. + * tablet where the end row and prev end row exactly match the given extent. */ Options forTablet(KeyExtent extent); /** - * Get the tablet metadata for the given extents. This will find tablets based on end row, so - * it's possible the prev rows could differ for the tablets returned. If this matters, then it - * must be checked. + * Get the tablet metadata for the given extents. This will only return tablets where the end + * row and prev end row exactly match the given extents. */ Options forTablets(Collection extents); @@ -431,8 +456,8 @@ public class TabletsMetadata implements Iterable, AutoCloseable private static class TabletMetadataIterator implements Iterator { private boolean sawLast = false; - private Iterator iter; - private Text endRow; + private final Iterator iter; + private final Text endRow; TabletMetadataIterator(Iterator source, Text endRow) { this.iter = source; @@ -487,9 +512,9 @@ public class TabletsMetadata implements Iterable, AutoCloseable .convertToTabletMetadata(); } - private ScannerBase scanner; + private final ScannerBase scanner; - private Iterable tablets; + private final Iterable tablets; private TabletsMetadata(TabletMetadata tm) { this.scanner = null; diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 709cc7f..591cf5a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -418,8 +418,9 @@ public class MetadataTableUtil { TabletMetadata tablet = context.getAmple().readTablet(extent, FILES, LOGS, PREV_ROW, DIR); - if (!tablet.getExtent().equals(extent)) - throw new RuntimeException("Unexpected extent " + tablet.getExtent() + " expected " + extent); + if (tablet == null) { + throw new RuntimeException("Tablet " + extent + " not found in metadata"); + } result.addAll(tablet.getLogs()); diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 41865f0..b63e1bd 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -190,8 +190,7 @@ public class Compactor extends AbstractServer implements CompactorService.Iface TabletMetadata tabletMeta = getContext().getAmple().readTablet(extent, ColumnType.ECOMP, ColumnType.PREV_ROW); - if (tabletMeta == null || !tabletMeta.getExtent().equals(extent) - || !tabletMeta.getExternalCompactions().containsKey(ecid)) { + if (tabletMeta == null || !tabletMeta.getExternalCompactions().containsKey(ecid)) { // table was deleted OR tablet was split or merged OR tablet no longer thinks compaction // is running for some reason LOG.info("Cancelling compaction {} that no longer has a metadata entry at {}", ecid, diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index c4bb1eb..1341838 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -926,6 +926,11 @@ public class TabletServer extends AbstractServer { static boolean checkTabletMetadata(KeyExtent extent, TServerInstance instance, TabletMetadata meta) throws AccumuloException { + if (meta == null) { + log.info("Not loading tablet {}, its metadata was not found.", extent); + return false; + } + if (!meta.sawPrevEndRow()) { throw new AccumuloException("Metadata entry does not have prev row (" + meta.getTableId() + " " + meta.getEndRow() + ")"); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 522efff..ad93e70 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -1361,9 +1361,8 @@ public class Tablet { var tabletMeta = context.getAmple().readTablet(extent, ColumnType.FILES, ColumnType.LOGS, ColumnType.ECOMP, ColumnType.PREV_ROW); - if (!tabletMeta.getExtent().equals(extent)) { - String msg = "Closed tablet " + extent + " does not match extent in metadata table " - + tabletMeta.getExtent(); + if (tabletMeta == null) { + String msg = "Closed tablet " + extent + " not found in metadata"; log.error(msg); throw new RuntimeException(msg); }