From commits-return-21881-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Wed May 30 14:40:04 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 7EA3718063B for ; Wed, 30 May 2018 14:40:03 +0200 (CEST) Received: (qmail 32416 invoked by uid 500); 30 May 2018 12:40:02 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 32407 invoked by uid 99); 30 May 2018 12:40:02 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 May 2018 12:40:02 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D30F480423; Wed, 30 May 2018 12:40:01 +0000 (UTC) Date: Wed, 30 May 2018 12:40:01 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch master updated: #469 Fix bug with GC not scanning root table for file refs (#503) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <152768400174.12098.17971787160174306938@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 5e7ea7853f6a1597e38d87c3ef272e1e9a695114 X-Git-Newrev: 780c4cab06597547d028ed59abdaa967a1c7900a X-Git-Rev: 780c4cab06597547d028ed59abdaa967a1c7900a X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/master by this push: new 780c4ca #469 Fix bug with GC not scanning root table for file refs (#503) 780c4ca is described below commit 780c4cab06597547d028ed59abdaa967a1c7900a Author: Keith Turner AuthorDate: Wed May 30 08:39:42 2018 -0400 #469 Fix bug with GC not scanning root table for file refs (#503) This bug was introduced in 9feb5e1. In addition to fixing the bug, specifying the table was made mandatory for the metadata scanner builder. The bug was found by MetadataMaxFilesIT --- .../core/metadata/schema/MetadataScanner.java | 20 ++++++++++++++++---- .../org/apache/accumulo/core/summary/Gatherer.java | 7 ++++--- .../apache/accumulo/gc/SimpleGarbageCollector.java | 4 ++-- .../accumulo/master/tableOps/bulkVer2/LoadFiles.java | 2 +- .../master/tableOps/bulkVer2/PrepBulkImport.java | 6 +++--- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java index 6fc05bb..573d3c4 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.client.impl.Table.ID; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; @@ -62,10 +63,11 @@ public class MetadataScanner implements Iterable, AutoCloseable TableOptions from(Connector conn); } - public interface TableOptions extends RangeOptions { - /** - * Optionally set a table name, defaults to {@value MetadataTable#NAME} - */ + public interface TableOptions { + RangeOptions scanRootTable(); + + RangeOptions scanMetadataTable(); + RangeOptions scanTable(String tableName); } @@ -310,6 +312,16 @@ public class MetadataScanner implements Iterable, AutoCloseable this.table = tableName; return this; } + + @Override + public RangeOptions scanMetadataTable() { + return scanTable(MetadataTable.NAME); + } + + @Override + public RangeOptions scanRootTable() { + return scanTable(RootTable.NAME); + } } private Scanner scanner; diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java index 0799510..eaae24d 100644 --- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java +++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java @@ -160,7 +160,7 @@ public class Gatherer { Predicate fileSelector) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { - Iterable tmi = MetadataScanner.builder().from(ctx) + Iterable tmi = MetadataScanner.builder().from(ctx).scanMetadataTable() .overRange(tableId, startRow, endRow).fetchFiles().fetchLocation().fetchLast().fetchPrev() .build(); @@ -522,8 +522,9 @@ public class Gatherer { private int countFiles() throws TableNotFoundException, AccumuloException, AccumuloSecurityException { // TODO use a batch scanner + iterator to parallelize counting files - return MetadataScanner.builder().from(ctx).overRange(tableId, startRow, endRow).fetchFiles() - .fetchPrev().build().stream().mapToInt(tm -> tm.getFiles().size()).sum(); + return MetadataScanner.builder().from(ctx).scanMetadataTable() + .overRange(tableId, startRow, endRow).fetchFiles().fetchPrev().build().stream() + .mapToInt(tm -> tm.getFiles().size()).sum(); } private class GatherRequest implements Supplier { diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 3c60861..6049b42 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -300,8 +300,8 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa throws TableNotFoundException, AccumuloException, AccumuloSecurityException { Stream tabletStream = MetadataScanner.builder().from(getConnector()) - .overTabletRange().checkConsistency().fetchDir().fetchFiles().fetchScans().build() - .stream(); + .scanTable(tableName).overTabletRange().checkConsistency().fetchDir().fetchFiles() + .fetchScans().build().stream(); Stream refStream = tabletStream.flatMap(tm -> { Stream refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream()) diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java index 1baac99..390a3ea 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java @@ -108,7 +108,7 @@ class LoadFiles extends MasterRepo { Text startRow = loadMapEntry.getKey().getPrevEndRow(); long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT); - Iterator tabletIter = MetadataScanner.builder().from(master) + Iterator tabletIter = MetadataScanner.builder().from(master).scanMetadataTable() .overRange(tableId, startRow, null).checkConsistency().fetchPrev().fetchLocation() .fetchLoaded().build().iterator(); diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java index 18ddab9..bca1389 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java @@ -166,9 +166,9 @@ public class PrepBulkImport extends MasterRepo { Iterators.transform(lmi, entry -> entry.getKey()); TabletIterFactory tabletIterFactory = startRow -> { - return MetadataScanner.builder().from(master).overRange(bulkInfo.tableId, startRow, null) - .checkConsistency().fetchPrev().build().stream().map(TabletMetadata::getExtent) - .iterator(); + return MetadataScanner.builder().from(master).scanMetadataTable() + .overRange(bulkInfo.tableId, startRow, null).checkConsistency().fetchPrev().build() + .stream().map(TabletMetadata::getExtent).iterator(); }; checkForMerge(bulkInfo.tableId.canonicalID(), -- To stop receiving notification emails like this one, please contact kturner@apache.org.